package at.jku.cp.old;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.commons.io.IOUtils;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
import org.jdom.xpath.XPath;

import at.jku.cp.mir.ArtistVector;
import at.jku.cp.mir.ArtistVectorOverlapScorSimilarity;
import at.jku.cp.mir.ArtistVectorSimilarity;
import at.jku.cp.mir.Stemmer;
import at.jku.cp.mir.Tag;

import com.google.common.collect.TreeMultimap;
import com.google.common.collect.TreeMultiset;
import com.google.common.collect.Multiset.Entry;

/**
 * This class normalizes tags of artists as described in
 * "THE QUEST FOR GROUND TRUTH IN MUSICAL ARTIST TAGGING IN THE SOCIAL WEB ERA"
 * by Geleijnse et al.
 * @author clemens
 */
public class TagNormalizer2
{

	/**
	 * Normalize the tags of given map, which contains artists and their tags.
	 * @param artistTagMap
	 * @return normalized artist-tag-map
	 */
	public Map<String, List<Tag>> normalizeTags(Map<String, List<Tag>> artistTagMap)
	{
		// this map contains all normalized artist tags
		Map<String, List<Tag>> normalizedArtistTagMap = new TreeMap<String, List<Tag>>();

		// this multiset contains all normalized tags for all artists and the
		// number of occurences in the artists
		TreeMultiset<String> normalizedTagNames = TreeMultiset.create();

		// this map contains the tags and their according artists
		TreeMultimap<String, String> tagArtistMap = TreeMultimap.create();

		// iterate over each artist and normalize the tags on an artist level
		Set<String> artists = artistTagMap.keySet();
		for (String artist : artists)
		{
			// normalize the artist name
			String normalizedArtistName = normalizeString(artist);

			// this set is used to check wether a tag with a given name has
			// already been normalized
			Set<String> normalizedTagSet = new TreeSet<String>();

			List<Tag> normalizedArtistTags = new LinkedList<Tag>();
			List<Tag> artistTags = artistTagMap.get(artist);
			for (Tag tag : artistTags)
			{
				// normalize the tag
				String tagName = tag.getName();
				String normalizedTagName = normalizeString(tagName);

				// check if the normalized tag is equal to the artist name
				boolean tagIsArtistName = normalizedArtistName.equalsIgnoreCase(normalizedTagName);
				boolean duplicateNormalizedTagName = normalizedTagSet.contains(normalizedTagName);
				if (!tagIsArtistName && tag.getCount() > 0 && !duplicateNormalizedTagName)
				{
					// add the tag to the normalized tag set
					normalizedTagSet.add(normalizedTagName);

					// add the tag to the normalized artist tags
					Tag normalizedTag = new Tag(normalizedTagName, tag.getCount());
					normalizedArtistTags.add(normalizedTag);

					// increase the counter for this tag in the global
					// normalized tag names map
					normalizedTagNames.add(normalizedTagName);

					// add this artist to the tagArtist map
					tagArtistMap.put(normalizedTagName, artist);
				}
			}

			int origTagCount = artistTags.size();
			int normalizedTagCount = normalizedArtistTags.size();

			if (artist.startsWith("Queens"))
			{
				System.out.println(artist + ": shortened tags by " + (origTagCount - normalizedTagCount)
						+ " so there are " + normalizedTagCount + " left: "
						+ Arrays.toString(normalizedArtistTags.toArray()));
			}

			// finally add the normalized artist tags to the new
			// normalizedArtistTagMap
			normalizedArtistTagMap.put(artist, normalizedArtistTags);
		}

		// the second step of normalization is to remove all tags which are
		// applied to less than 5% of the artists --> remove all tags in the
		// normalized tag multiset wich do not occur in at least 5% of the
		// artists
		int fivePercentOfArtists = (int) Math.abs(artists.size() * 0.05);
		Set<Entry<String>> tagNameEntrySet = normalizedTagNames.entrySet();
		List<String> elementsToRemove = new LinkedList<String>();
		for (Entry<String> entry : tagNameEntrySet)
		{
			if (entry.getCount() < fivePercentOfArtists)
			{
				// add this tag to the tags which shall be removed
				String tagName = entry.getElement();
				elementsToRemove.add(tagName);

				// get all artists which have this tag
				SortedSet<String> artistNames = tagArtistMap.get(tagName);
				for (String artist : artistNames)
				{
					List<Tag> artistTags = normalizedArtistTagMap.get(artist);
					Tag tagToRemove = null;
					for (Tag artistTag : artistTags)
					{
						if (artistTag.getName().equalsIgnoreCase(tagName))
						{
							tagToRemove = artistTag;
							break;
						}
					}
					artistTags.remove(tagToRemove);
				}
			}
		}
		normalizedTagNames.removeAll(elementsToRemove);
		System.out.println("removed " + elementsToRemove.size() + " tags; so there are still "
				+ normalizedTagNames.elementSet().size() + " unique tags for " + artists.size() + " artists");

		// second part: normalize artist tag by looking at the 10 top tracks of
		// the artist
		for (String artist : artists)
		{
			List<Tag> artistTags = normalizedArtistTagMap.get(artist);
			List<Tag> filteredTags;
			try
			{
				filteredTags = filterArtistTracks(artist, artistTags);
			} catch (Exception e)
			{
				e.printStackTrace();
				continue;
			}
			// System.out.println(artist + " --> previous tags: " +
			// Arrays.toString(artistTags.toArray()));
			// System.out.println(artist + " --> filtered tags: " +
			// Arrays.toString(filteredTags.toArray()));
			artistTagMap.put(artist, filteredTags);
		}

		// do the TF-IDF calculation
		int D = artists.size();
		int count = 0;
		SortedSet<String> tagNames = normalizedTagNames.elementSet();
		Map<String, ArtistVector> artistVectors = new TreeMap<String, ArtistVector>();
		for (String tagName : tagNames)
		{
			// System.out.println(count + " analyzing tag: " + tagName);

			// get all artists which have this tag
			SortedSet<String> artistsForThisTag = tagArtistMap.get(tagName);
			double nrOfArtistsWithThisTag = artistsForThisTag.size();

			for (String artist : artistsForThisTag)
			{
				int n = getTagCount(normalizedArtistTagMap.get(artist), tagName);
				double nrOfTagsForThisArtist = artistTagMap.get(artist).size();
				double tf = n / nrOfTagsForThisArtist;
				double idf = Math.log(D / (1 + nrOfArtistsWithThisTag));

				double tfidf = tf * idf;

				ArtistVector aV = artistVectors.get(artist);
				if (aV == null)
				{
					aV = new ArtistVector(artist);
					artistVectors.put(artist, aV);
				}
				aV.addTagTfIdf(tagName, tfidf);
			}

			count++;
		}

		Iterator<String> artistIt = artists.iterator();
		ArtistVectorSimilarity avs = new ArtistVectorOverlapScorSimilarity();
		TreeMultimap<String, String> artistSimilarityMap = TreeMultimap.create();
		while (artistIt.hasNext())
		{
			String currentArtist = artistIt.next();
			ArtistVector firstArtistVector = artistVectors.get(currentArtist);

			Iterator<String> secondArtistIt = artists.iterator();
			TreeMultimap<Double, String> similarityMap = TreeMultimap.create();
			while (secondArtistIt.hasNext())
			{
				String secondArtist = secondArtistIt.next();
				if (!currentArtist.equals(secondArtist))
				{
					ArtistVector av = artistVectors.get(secondArtist);
					double similarity = avs.getSimilarity(firstArtistVector, av);
					similarityMap.put(Double.valueOf(similarity), secondArtist);
				}
			}

			SortedSet<Double> keySet = similarityMap.keySet();
			Double similarity = keySet.last();
			SortedSet<String> similarArtists = similarityMap.get(similarity);
			System.out.println(currentArtist + " --> similar artists (by " + similarity + "): "
					+ Arrays.toString(similarArtists.toArray()));

			artistSimilarityMap.putAll(currentArtist, similarArtists);

		}

		writeSimilarArtistFile(artistSimilarityMap);

		return null;
	}

	private void writeSimilarArtistFile(TreeMultimap<String, String> artistSimilarityMap)
	{

		try
		{
			File similarityMapFile = new File("similarity.txt");
			OutputStream outStream = new FileOutputStream(similarityMapFile);

			File artistFile = new File("C224a.txt");

			Map<String, String> lastFmArtistDict = new TreeMap<String, String>();

			Scanner scanner = new Scanner(artistFile);
			while (scanner.hasNextLine())
			{
				String artistLine = scanner.nextLine();
				String origArtist = artistLine;
				String lastFmArtist = artistLine;
				if (artistLine.contains(" --> "))
				{
					String[] split = artistLine.split(" --> ");
					origArtist = split[0];
					lastFmArtist = split[1];
				}

				lastFmArtistDict.put(lastFmArtist, origArtist);
			}

			Set<String> lastFmArtists = lastFmArtistDict.keySet();
			for (String lastFmArtist : lastFmArtists)
			{
				String origArtist = lastFmArtistDict.get(lastFmArtist);
				SortedSet<String> similarArtists = artistSimilarityMap.get(lastFmArtist);

				StringBuilder artistString = new StringBuilder(origArtist);
				artistString.append(" --> ");
				for (String similarLastFmArtist : similarArtists)
				{
					artistString.append(lastFmArtistDict.get(similarLastFmArtist)).append(",");
				}
				StringBuilder line = artistString.reverse().delete(0, 1).reverse();
				IOUtils.write(line.toString(), outStream);
				IOUtils.write("\n", outStream);
			}

		} catch (FileNotFoundException e)
		{
			e.printStackTrace();
		} catch (IOException e)
		{
			e.printStackTrace();
		}
	}

	private int getTagCount(List<Tag> list, String tagName)
	{
		for (Tag tag : list)
		{
			if (tag.getName().equalsIgnoreCase(tagName))
				return tag.getCount();
		}

		return 0;
	}

	private List<Tag> filterArtistTracks(String artist, List<Tag> artistTags) throws Exception
	{
		// get a list of the tags for the top tracks of this artist
		String artistName = artist.replace("/", "_");
		File artistFolder = new File("artistTags3", artistName);
		File[] tagFiles = artistFolder.listFiles();

		SAXBuilder builder = new SAXBuilder();
		XPath tagXPath = XPath.newInstance("//tag/name");

		int fileCount = 0;

		TreeMultiset<String> artistTrackTags = TreeMultiset.create();
		for (File tagFile : tagFiles)
		{
			Document tagDoc = builder.build(tagFile);
			List<Element> tagElements = tagXPath.selectNodes(tagDoc);
			if (tagElements.size() > 1)
			{
				// only add the tags of a song has more than 4 tags ...
				fileCount++;
				for (Element element : tagElements)
				{
					String tagName = element.getTextTrim();
					String normalizedTagName = normalizeString(tagName);
					artistTrackTags.add(normalizedTagName);
				}
			}
		}

		// at least 30 percent
		int minCount = (int) Math.abs(fileCount * 0.3);

		List<Tag> filteredArtistTracks = new LinkedList<Tag>();
		for (Tag tag : artistTags)
		{
			String tagName = tag.getName();
			int count = artistTrackTags.count(tagName);
			if (count >= minCount)
			{
				filteredArtistTracks.add(tag);
			}
		}

		return filteredArtistTracks;
	}

	private String normalizeString(String s)
	{
		if (s == null)
			return null;

		// turn the string into lower case
		String tmp = s.toLowerCase();

		// TODO replace all á, à, ... with a's, and so on for all letters with
		// apostrophes

		// stemm the word(s)
		Stemmer st;
		String[] words = tmp.split("[ ,-/]");
		StringBuilder sb = new StringBuilder();
		for (String word : words)
		{
			st = new Stemmer();
			st.add(word.toCharArray(), word.length());
			st.stem();
			sb.append(st.toString()).append(" ");
		}

		// replace all non letter and all non digit characters inside the string
		// with an empty character
		String normalized = sb.toString().replaceAll("[^a-z0-9]", "");
		return normalized;
	}

}
